import redis
import json
from tour_comment_crawler.settings import REDIS_DB, REDIS_HOST, REDIS_PASSWORD, REDIS_PORT

r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD)
local_debug = False

crawlall = True

area = "xianju"
category = "sight"
for key in r.hgetall(area + ":" + category):
    value = r.hget(area + ":" + category, key).decode()
    value = json.loads(value)
    print(key, value['ota_info'])
    for ota_info in value['ota_info']:
        url_key = ota_info['ota'] + ":start_urls"
        url = ota_info['request_url']
        if ota_info['ota'] == "xiecheng_piao":
            r.sadd(url_key, url)
